{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/cpu/lib/python3.6/site-packages/gym/core.py:26: UserWarning: \u001b[33mWARN: Gym minimally supports python 3.6 as the python foundation not longer supports the version, please update your version to 3.7+\u001b[0m\n",
      "  \"Gym minimally supports python 3.6 as the python foundation not longer supports the version, please update your version to 3.7+\"\n",
      "/root/anaconda3/envs/cpu/lib/python3.6/site-packages/gym/envs/registration.py:593: UserWarning: \u001b[33mWARN: The environment CartPole-v0 is out of date. You should consider upgrading to version `v1`.\u001b[0m\n",
      "  f\"The environment {id} is out of date. You should consider \"\n",
      "/root/anaconda3/envs/cpu/lib/python3.6/site-packages/gym/core.py:330: DeprecationWarning: \u001b[33mWARN: Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
      "  \"Initializing wrapper in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\"\n",
      "/root/anaconda3/envs/cpu/lib/python3.6/site-packages/gym/wrappers/step_api_compatibility.py:40: DeprecationWarning: \u001b[33mWARN: Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\u001b[0m\n",
      "  \"Initializing environment in old step API which returns one bool instead of two. It is recommended to set `new_step_api=True` to use new step API. This will be the default behaviour in future.\"\n",
      "/root/anaconda3/envs/cpu/lib/python3.6/site-packages/gym/core.py:52: DeprecationWarning: \u001b[33mWARN: The argument mode in render method is deprecated; use render_mode during environment initialization instead.\n",
      "See here for more information: https://www.gymlibrary.ml/content/api/\u001b[0m\n",
      "  \"The argument mode in render method is deprecated; \"\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAASf0lEQVR4nO3db4xdd53f8fcnE8dJcRYcMk69trNxqZFI0K4TjVykLKsU6MZNqxpUpXKkIj+IZCSMBOqKNtmVduGBpW21QJ8UJFMiLMoSjCCKQWm7XhdKULcxDjjBjuONg73JYCfjhLJJ2OD4z7cP5li52DOe6/mz49/c90u6uud8z+/c+/1Zzscnvzl3bqoKSVI7rpjvBiRJl8bglqTGGNyS1BiDW5IaY3BLUmMMbklqzJwFd5L1SQ4lOZzkvrl6H0kaNJmL+7iTDAF/DfwzYBT4IXBPVT01628mSQNmrq641wGHq+qnVfUG8CCwYY7eS5IGypVz9LorgOd79keBfzLZ4Ouvv75uuummOWpFktpz9OhRXnrppUx0bK6Ce6I3+7U1mSSbgc0AN954I3v37p2jViSpPSMjI5Mem6ulklFgVc/+SuBY74Cq2lZVI1U1Mjw8PEdtSNLCM1fB/UNgTZLVSa4CNgI75+i9JGmgzMlSSVWdTvIx4H8CQ8ADVXVgLt5LkgbNXK1xU1WPAI/M1etL0qDyk5OS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhozo68uS3IUeBU4A5yuqpEk1wFfB24CjgL/pqr+38zalCSdMxtX3P+0qtZW1Ui3fx+wu6rWALu7fUnSLJmLpZINwPZuezvwwTl4D0kaWDMN7gL+IsnjSTZ3tRuq6jhA97xshu8hSeoxozVu4PaqOpZkGbArydP9ntgF/WaAG2+8cYZtSNLgmNEVd1Ud657HgIeAdcCLSZYDdM9jk5y7rapGqmpkeHh4Jm1I0kCZdnAneUuSa89tA78P7Ad2Apu6YZuAh2fapCTpTTNZKrkBeCjJudf586r6H0l+COxIci/wHHD3zNuUJJ0z7eCuqp8CvzNB/WXg/TNpSpI0OT85KUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDVmyuBO8kCSsST7e2rXJdmV5JnueWnPsfuTHE5yKMmdc9W4JA2qfq64vwysP692H7C7qtYAu7t9ktwMbARu6c75fJKhWetWkjR1cFfV94Gfn1feAGzvtrcDH+ypP1hVJ6vqCHAYWDc7rUqSYPpr3DdU1XGA7nlZV18BPN8zbrSrXSDJ5iR7k+w9ceLENNuQpMEz2z+czAS1mmhgVW2rqpGqGhkeHp7lNiRp4ZpucL+YZDlA9zzW1UeBVT3jVgLHpt+eJOl80w3uncCmbnsT8HBPfWOSxUlWA2uAPTNrUZLU68qpBiT5GnAHcH2SUeBPgD8FdiS5F3gOuBugqg4k2QE8BZwGtlTVmTnqXZIG0pTBXVX3THLo/ZOM3wpsnUlTkqTJ+clJSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNmTK4kzyQZCzJ/p7ap5L8LMm+7nFXz7H7kxxOcijJnXPVuCQNqn6uuL8MrJ+g/rmqWts9HgFIcjOwEbilO+fzSYZmq1lJUh/BXVXfB37e5+ttAB6sqpNVdQQ4DKybQX+SpPPMZI37Y0me7JZSlna1FcDzPWNGu9oFkmxOsjfJ3hMnTsygDUkaLNMN7i8A7wDWAseBz3T1TDC2JnqBqtpWVSNVNTI8PDzNNiRp8EwruKvqxao6U1VngS/y5nLIKLCqZ+hK4NjMWpQk9ZpWcCdZ3rP7IeDcHSc7gY1JFidZDawB9sysRUlSryunGpDka8AdwPVJRoE/Ae5IspbxZZCjwEcAqupAkh3AU8BpYEtVnZmTziVpQE0Z3FV1zwTlL11k/FZg60yakiRNzk9OSlJjDG5JaozBLUmNMbglqTEGtyQ1Zsq7SqSFqqr45dgRzp5+44Jjb1m2mqFFi+ehK2lqBrcGVxVHvvdlfvWLF369nvDuuz/NNUv/4fz0JU3BpRJJaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxUwZ3klVJvpvkYJIDST7e1a9LsivJM93z0p5z7k9yOMmhJHfO5QQkadD0c8V9GviDqnoX8B5gS5KbgfuA3VW1Btjd7dMd2wjcAqwHPp9kaC6al6RBNGVwV9XxqvpRt/0qcBBYAWwAtnfDtgMf7LY3AA9W1cmqOgIcBtbNct+SNLAuaY07yU3ArcBjwA1VdRzGwx1Y1g1bATzfc9poVzv/tTYn2Ztk74kTJ6bRuiQNpr6DO8kS4JvAJ6rqlYsNnaBWFxSqtlXVSFWNDA8P99uGJA28voI7ySLGQ/urVfWtrvxikuXd8eXAWFcfBVb1nL4SODY77UqS+rmrJMCXgINV9dmeQzuBTd32JuDhnvrGJIuTrAbWAHtmr2VJGmz9fHXZ7cCHgZ8k2dfV/hD4U2BHknuB54C7AarqQJIdwFOM35GyparOzHbjkjSopgzuqvoBE69bA7x/knO2Altn0JckaRJ+clKSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmP6+bLgVUm+m+RgkgNJPt7VP5XkZ0n2dY+7es65P8nhJIeS3DmXE5CkQdPPlwWfBv6gqn6U5Frg8SS7umOfq6o/6x2c5GZgI3AL8JvAXyZ5p18YLEmzY8or7qo6XlU/6rZfBQ4CKy5yygbgwao6WVVHgMPAutloVpJ0iWvcSW4CbgUe60ofS/JkkgeSLO1qK4Dne04b5eJBL0m6BH0Hd5IlwDeBT1TVK8AXgHcAa4HjwGfODZ3g9Jrg9TYn2Ztk74kTJy61b0kaWH0Fd5JFjIf2V6vqWwBV9WJVnamqs8AXeXM5ZBRY1XP6SuDY+a9ZVduqaqSqRoaHh2cyB0kaKP3cVRLgS8DBqvpsT315z7APAfu77Z3AxiSLk6wG1gB7Zq9lSRps/dxVcjvwYeAnSfZ1tT8E7kmylvFlkKPARwCq6kCSHcBTjN+RssU7SiRp9kwZ3FX1AyZet37kIudsBbbOoC9J0iT85KQkNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGtPPbweUmrJ//35eeeWVKceFYuj11y/4DWpVsG/fj6nFb53yNYaGhrjttttYtGjRNLuVLp3BrQXnox/9KI8++uiU465I+Nof/2tWL1/6a/WzZ89wzz338Dcv/u2Ur7FkyRKeffZZli1bNu1+pUtlcGvg/fLMtRz71Ts5zZXccNXf8Nah4/PdknRRBrcGVgGvnr6O5/72Lv7u7G8A8Pyv3sW7/sGj1IS/gl66PPjDSQ20n7z2e/zd2bcy/l0h4UxdxYHXfpfXz/zGfLcmTcrg1gAbD+rznakrveLWZa2fLwu+OsmeJE8kOZDk0139uiS7kjzTPS/tOef+JIeTHEpy51xOQJq+4uqhVy+oLr7ida6IX5Oqy1c/V9wngfdV1e8Aa4H1Sd4D3Afsrqo1wO5unyQ3AxuBW4D1wOeTDM1B79KMBPjtJf+b6xb9jHAGOMs1V7zC2mt3c80VFwa6dLno58uCC3it213UPQrYANzR1bcD3wP+Q1d/sKpOAkeSHAbWAX812XucOnWKF154YXozkM7zxhtv9DXubBXbH3mUa67ex0unVnC2rmTpohf4P1e8ysuvvN7Xa1QVY2NjnD17diYtSxc4derUpMf6uquku2J+HPjHwH+pqseS3FBVxwGq6niSczeyrgD+b8/po11tUi+//DJf+cpX+mlFmtLY2FjfY3c/fqTbenJa73Xq1Cm+8Y1vsGTJkmmdL03m5ZdfnvRYX8FdVWeAtUneBjyU5N0XGT7RT3XqgkHJZmAzwI033sgnP/nJflqRpvTtb3+bI0eOTD1wFlx11VVs2bLFD+Bo1n3961+f9Ngl3VVSVb9gfElkPfBikuUA3fO5y5xRYFXPaSuBYxO81raqGqmqkeHh4UtpQ5IGWj93lQx3V9okuQb4APA0sBPY1A3bBDzcbe8ENiZZnGQ1sAbYM8t9S9LA6mepZDmwvVvnvgLYUVXfSfJXwI4k9wLPAXcDVNWBJDuAp4DTwJZuqUWSNAv6uavkSeDWCeovA++f5JytwNYZdydJuoCfnJSkxhjcktQYfzugFpz3vve9vP3tb/97ea+rr76axYsX/728l3SOwa0FZ+tWf7yihc2lEklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUmH6+LPjqJHuSPJHkQJJPd/VPJflZkn3d466ec+5PcjjJoSR3zuUEJGnQ9PP7uE8C76uq15IsAn6Q5L93xz5XVX/WOzjJzcBG4BbgN4G/TPJOvzBYkmbHlFfcNe61bndR96iLnLIBeLCqTlbVEeAwsG7GnUqSgD7XuJMMJdkHjAG7quqx7tDHkjyZ5IEkS7vaCuD5ntNHu5okaRb0FdxVdaaq1gIrgXVJ3g18AXgHsBY4DnymG56JXuL8QpLNSfYm2XvixIlptC5Jg+mS7iqpql8A3wPWV9WLXaCfBb7Im8sho8CqntNWAscmeK1tVTVSVSPDw8PT6V2SBlI/d5UMJ3lbt30N8AHg6STLe4Z9CNjfbe8ENiZZnGQ1sAbYM6tdS9IA6+eukuXA9iRDjAf9jqr6TpKvJFnL+DLIUeAjAFV1IMkO4CngNLDFO0okafZMGdxV9SRw6wT1D1/knK3A1pm1JkmaiJ+clKTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjUlVzXcPJDkB/BJ4ab57mQPX47xas1Dn5rza8ltVNTzRgcsiuAGS7K2qkfnuY7Y5r/Ys1Lk5r4XDpRJJaozBLUmNuZyCe9t8NzBHnFd7FurcnNcCcdmscUuS+nM5XXFLkvow78GdZH2SQ0kOJ7lvvvu5VEkeSDKWZH9P7boku5I80z0v7Tl2fzfXQ0nunJ+up5ZkVZLvJjmY5ECSj3f1pueW5Ooke5I80c3r01296Xmdk2QoyY+TfKfbXyjzOprkJ0n2Jdnb1RbE3KalqubtAQwBzwL/CLgKeAK4eT57msYcfg+4DdjfU/tPwH3d9n3Af+y2b+7muBhY3c19aL7nMMm8lgO3ddvXAn/d9d/03IAAS7rtRcBjwHtan1fP/P4d8OfAdxbK38Wu36PA9efVFsTcpvOY7yvudcDhqvppVb0BPAhsmOeeLklVfR/4+XnlDcD2bns78MGe+oNVdbKqjgCHGf8zuOxU1fGq+lG3/SpwEFhB43Orca91u4u6R9H4vACSrAT+BfBfe8rNz+siFvLcLmq+g3sF8HzP/mhXa90NVXUcxgMQWNbVm5xvkpuAWxm/Om1+bt1ywj5gDNhVVQtiXsB/Bv49cLanthDmBeP/uP5FkseTbO5qC2Vul+zKeX7/TFBbyLe5NDffJEuAbwKfqKpXkommMD50gtplObeqOgOsTfI24KEk777I8CbmleRfAmNV9XiSO/o5ZYLaZTevHrdX1bEky4BdSZ6+yNjW5nbJ5vuKexRY1bO/Ejg2T73MpheTLAfonse6elPzTbKI8dD+alV9qysviLkBVNUvgO8B62l/XrcD/yrJUcaXHN+X5L/R/rwAqKpj3fMY8BDjSx8LYm7TMd/B/UNgTZLVSa4CNgI757mn2bAT2NRtbwIe7qlvTLI4yWpgDbBnHvqbUsYvrb8EHKyqz/YcanpuSYa7K22SXAN8AHiaxudVVfdX1cqquonx/47+V1X9WxqfF0CStyS59tw28PvAfhbA3KZtvn86CtzF+B0LzwJ/NN/9TKP/rwHHgVOM/0t/L/B2YDfwTPd8Xc/4P+rmegj45/Pd/0Xm9buM/+/lk8C+7nFX63MDfhv4cTev/cAfd/Wm53XeHO/gzbtKmp8X43edPdE9DpzLiYUwt+k+/OSkJDVmvpdKJEmXyOCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4Jakx/x/6BHhB28nxIgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "#创建环境\n",
    "env = gym.make('CartPole-v0')\n",
    "env.reset()\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.imshow(env.render(mode='rgb_array'))\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, tensor([[0.5369, 0.4631]], grad_fn=<SoftmaxBackward0>))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "import random\n",
    "\n",
    "\n",
    "#定义模型\n",
    "class Model(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.sequential = torch.nn.Sequential(\n",
    "            torch.nn.Linear(4, 128),\n",
    "            torch.nn.ReLU(),\n",
    "            torch.nn.Linear(128, 2),\n",
    "            torch.nn.Softmax(dim=1),\n",
    "        )\n",
    "\n",
    "    #得到一个动作\n",
    "    def forward(self, state):\n",
    "        #[1, 4] -> [1, 2]\n",
    "        prob = self.sequential(state)\n",
    "\n",
    "        #根据概率选择一个动作\n",
    "        action = random.choices(range(2), weights=prob[0].tolist(), k=1)[0]\n",
    "\n",
    "        return action, prob\n",
    "\n",
    "\n",
    "model = Model()\n",
    "\n",
    "model(torch.randn(1, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.2256]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_td = sequential = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 1),\n",
    ")\n",
    "\n",
    "model_td(torch.randn(1, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([tensor([[-0.0388, -0.0149,  0.0431, -0.0149]]),\n",
       "  tensor([[-0.0391,  0.1796,  0.0428, -0.2937]]),\n",
       "  tensor([[-0.0355,  0.3741,  0.0369, -0.5726]]),\n",
       "  tensor([[-0.0281,  0.5687,  0.0255, -0.8534]]),\n",
       "  tensor([[-0.0167,  0.3732,  0.0084, -0.5528]]),\n",
       "  tensor([[-0.0092,  0.1780, -0.0026, -0.2575]]),\n",
       "  tensor([[-0.0057,  0.3731, -0.0078, -0.5510]]),\n",
       "  tensor([[ 0.0018,  0.1781, -0.0188, -0.2608]]),\n",
       "  tensor([[ 0.0054, -0.0167, -0.0240,  0.0259]]),\n",
       "  tensor([[ 0.0050, -0.2115, -0.0235,  0.3109]]),\n",
       "  tensor([[ 0.0008, -0.4063, -0.0173,  0.5960]]),\n",
       "  tensor([[-0.0073, -0.2109, -0.0054,  0.2980]]),\n",
       "  tensor([[-1.1553e-02, -4.0595e-01,  5.7758e-04,  5.8894e-01]]),\n",
       "  tensor([[-0.0197, -0.6011,  0.0124,  0.8818]]),\n",
       "  tensor([[-0.0317, -0.4061,  0.0300,  0.5930]]),\n",
       "  tensor([[-0.0398, -0.6017,  0.0419,  0.8950]]),\n",
       "  tensor([[-0.0518, -0.7973,  0.0598,  1.2005]]),\n",
       "  tensor([[-0.0678, -0.9932,  0.0838,  1.5113]]),\n",
       "  tensor([[-0.0877, -1.1892,  0.1140,  1.8290]]),\n",
       "  tensor([[-0.1114, -0.9955,  0.1506,  1.5737]]),\n",
       "  tensor([[-0.1314, -1.1921,  0.1820,  1.9094]])],\n",
       " [1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0,\n",
       "  1.0],\n",
       " [1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0],\n",
       " [tensor([[0.5619, 0.4381]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5635, 0.4365]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5589, 0.4411]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5648, 0.4352]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5575, 0.4425]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5622, 0.4378]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5565, 0.4435]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5615, 0.4385]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5595, 0.4405]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5536, 0.4464]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5426, 0.4574]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5541, 0.4459]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5430, 0.4570]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5378, 0.4622]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5440, 0.4560]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5387, 0.4613]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5343, 0.4657]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5363, 0.4637]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5405, 0.4595]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5383, 0.4617]], grad_fn=<SoftmaxBackward0>),\n",
       "  tensor([[0.5422, 0.4578]], grad_fn=<SoftmaxBackward0>)],\n",
       " [tensor([[-0.0391,  0.1796,  0.0428, -0.2937]]),\n",
       "  tensor([[-0.0355,  0.3741,  0.0369, -0.5726]]),\n",
       "  tensor([[-0.0281,  0.5687,  0.0255, -0.8534]]),\n",
       "  tensor([[-0.0167,  0.3732,  0.0084, -0.5528]]),\n",
       "  tensor([[-0.0092,  0.1780, -0.0026, -0.2575]]),\n",
       "  tensor([[-0.0057,  0.3731, -0.0078, -0.5510]]),\n",
       "  tensor([[ 0.0018,  0.1781, -0.0188, -0.2608]]),\n",
       "  tensor([[ 0.0054, -0.0167, -0.0240,  0.0259]]),\n",
       "  tensor([[ 0.0050, -0.2115, -0.0235,  0.3109]]),\n",
       "  tensor([[ 0.0008, -0.4063, -0.0173,  0.5960]]),\n",
       "  tensor([[-0.0073, -0.2109, -0.0054,  0.2980]]),\n",
       "  tensor([[-1.1553e-02, -4.0595e-01,  5.7758e-04,  5.8894e-01]]),\n",
       "  tensor([[-0.0197, -0.6011,  0.0124,  0.8818]]),\n",
       "  tensor([[-0.0317, -0.4061,  0.0300,  0.5930]]),\n",
       "  tensor([[-0.0398, -0.6017,  0.0419,  0.8950]]),\n",
       "  tensor([[-0.0518, -0.7973,  0.0598,  1.2005]]),\n",
       "  tensor([[-0.0678, -0.9932,  0.0838,  1.5113]]),\n",
       "  tensor([[-0.0877, -1.1892,  0.1140,  1.8290]]),\n",
       "  tensor([[-0.1114, -0.9955,  0.1506,  1.5737]]),\n",
       "  tensor([[-0.1314, -1.1921,  0.1820,  1.9094]]),\n",
       "  tensor([[-0.1552, -1.3886,  0.2202,  2.2525]])],\n",
       " [False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  False,\n",
       "  True])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_data():\n",
    "    states = []\n",
    "    rewards = []\n",
    "    actions = []\n",
    "    probs = []\n",
    "    next_states = []\n",
    "    overs = []\n",
    "\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        state = torch.FloatTensor(state).reshape(1, 4)\n",
    "        action, prob = model(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        next_state, reward, over, _ = env.step(action)\n",
    "\n",
    "        #记录数据样本\n",
    "        states.append(state)\n",
    "        rewards.append(reward)\n",
    "        actions.append(action)\n",
    "        probs.append(prob)\n",
    "        next_states.append(torch.FloatTensor(next_state).reshape(1, 4))\n",
    "        overs.append(over)\n",
    "\n",
    "        #更新游戏状态,开始下一个动作\n",
    "        state = next_state\n",
    "\n",
    "    return states, rewards, actions, probs, next_states, overs\n",
    "\n",
    "\n",
    "states, rewards, actions, probs, next_states, overs = get_data()\n",
    "\n",
    "states, rewards, actions, probs, next_states, overs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12.0"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "\n",
    "\n",
    "def test(play):\n",
    "    #初始化游戏\n",
    "    state = env.reset()\n",
    "\n",
    "    #记录反馈值的和,这个值越大越好\n",
    "    reward_sum = 0\n",
    "\n",
    "    #玩到游戏结束为止\n",
    "    over = False\n",
    "    while not over:\n",
    "        #根据当前状态得到一个动作\n",
    "        state = torch.FloatTensor(state).reshape(1, 4)\n",
    "        with torch.no_grad():\n",
    "            action, _ = model(state)\n",
    "\n",
    "        #执行动作,得到反馈\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum += reward\n",
    "\n",
    "        #打印动画\n",
    "        if play:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    return reward_sum\n",
    "\n",
    "\n",
    "test(play=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[8.090483997483998, 8.690100963999999, 8.260044, 6.724, 4.0]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_advantages(deltas):\n",
    "    advantages = []\n",
    "    advantage = 0.0\n",
    "    for delta in deltas[::-1]:\n",
    "        advantage = 0.98 * 0.95 * advantage + delta\n",
    "        advantages.append(advantage)\n",
    "    advantages.reverse()\n",
    "    return advantages\n",
    "\n",
    "\n",
    "get_advantages(list(range(5)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.0207, grad_fn=<DivBackward0>)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def compute_surrogate_obj(states, actions, advantages, old_logs, model):\n",
    "    new_logs = []\n",
    "    for state, action in zip(states, actions):\n",
    "        _, prob = model(state)\n",
    "        new_logs.append(prob[0, action].log())\n",
    "\n",
    "    ratios = [(new_log - old_log).exp()\n",
    "              for new_log, old_log in zip(new_logs, old_logs)]\n",
    "    ratios = [\n",
    "        ratio * advantage for ratio, advantage in zip(ratios, advantages)\n",
    "    ]\n",
    "    return sum(ratios) / len(ratios)\n",
    "\n",
    "\n",
    "compute_surrogate_obj(states, actions, list(range(len(states))),\n",
    "                      list(range(len(states))), model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "executionInfo": {
     "elapsed": 8251,
     "status": "ok",
     "timestamp": 1650011468229,
     "user": {
      "displayName": "Sam Lu",
      "userId": "15789059763790170725"
     },
     "user_tz": -480
    },
    "id": "BQXVYW2T_DcQ",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 2.1828e-11, -1.7508e-10, -1.4188e-10, -1.0532e-09, -7.2760e-11,\n",
      "        -6.9849e-10,  4.5111e-10,  6.9849e-10, -2.5466e-11, -9.3132e-10,\n",
      "        -1.1642e-10,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -1.0914e-11,  5.6207e-10,  1.2005e-10, -3.1650e-10,\n",
      "        -9.4587e-11,  4.1837e-11, -4.2928e-10,  4.9477e-10,  1.0914e-11,\n",
      "        -3.9290e-10, -4.6566e-10,  8.7311e-10, -3.6380e-11,  1.3079e-09,\n",
      "         7.2760e-12,  4.2201e-10,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  4.0018e-11, -4.5475e-12,  2.5830e-10, -2.0736e-09,\n",
      "        -1.0914e-10,  3.6016e-10, -2.7649e-10,  2.2337e-09,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -1.4552e-11, -1.4916e-10,\n",
      "         6.1846e-11, -4.1473e-10,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -7.2760e-12, -2.3283e-10, -3.0559e-10, -1.1059e-09,\n",
      "         4.3656e-11,  6.6211e-10, -2.4011e-10,  2.3429e-09,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  1.0186e-10, -4.0509e-09,\n",
      "        -4.3656e-11,  3.3469e-09,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -2.1828e-11,  2.0063e-09,  4.9477e-10, -1.4261e-09,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -1.4552e-11,\n",
      "        -1.5189e-10,  3.0559e-10, -1.1059e-09, -1.6735e-10, -1.1605e-09,\n",
      "         7.1304e-10, -1.1642e-10,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  6.2573e-10,  5.8208e-11,  1.2224e-09,  5.0932e-11,\n",
      "         6.3119e-10,  5.5297e-10, -1.5716e-09,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -9.0949e-13,  1.1369e-11, -1.2733e-11,  9.0949e-12,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -1.9327e-12,\n",
      "        -8.6970e-12,  1.2506e-11, -2.4215e-11,  7.2760e-12, -4.6566e-10,\n",
      "         1.6735e-10, -4.6566e-10, -5.8208e-11, -5.9845e-10,  1.6735e-10,\n",
      "        -6.2209e-10,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -8.0036e-11, -6.3665e-11, -1.7462e-10, -8.8767e-10,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -6.5484e-11, -6.5484e-11,\n",
      "        -1.7462e-10,  5.0131e-09,  7.2760e-12,  2.3647e-11, -2.4011e-10,\n",
      "        -1.3097e-10,  6.3665e-12,  9.2768e-11, -7.2760e-12, -2.6739e-10,\n",
      "        -3.6380e-12, -5.6298e-10,  8.0036e-11,  3.0559e-10, -4.5475e-12,\n",
      "         6.9122e-11, -2.5466e-11, -8.3674e-11,  1.2733e-11, -2.9650e-10,\n",
      "         1.3461e-10,  1.8008e-10,  0.0000e+00,  0.0000e+00,  1.4552e-11,\n",
      "        -5.0932e-11,  1.8190e-12, -1.1642e-10,  7.2760e-12, -1.7462e-10,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  3.2742e-11, -5.1841e-10,\n",
      "        -9.0949e-11, -2.0736e-09,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -1.0914e-11,  1.9736e-10, -6.9122e-11, -5.1659e-10,\n",
      "        -2.1828e-11,  0.0000e+00, -8.7311e-11, -5.8208e-11,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -1.6007e-10, -4.4747e-10,\n",
      "         1.8917e-10, -4.1036e-09, -5.4570e-11,  1.0186e-10, -1.2733e-10,\n",
      "        -1.5971e-09,  3.6380e-11, -1.8190e-11, -8.7311e-11,  1.8917e-10,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -2.7285e-12,  2.7649e-10, -1.6371e-11,  5.5297e-10,\n",
      "        -1.8190e-12,  4.5475e-13,  0.0000e+00,  0.0000e+00,  4.0018e-11,\n",
      "        -1.1441e-09,  1.8190e-10,  8.6584e-10,  2.5466e-11, -1.0350e-09,\n",
      "        -8.3674e-11, -1.9645e-10, -1.0186e-10,  9.4224e-10,  4.6566e-10,\n",
      "         6.6211e-09, -8.5493e-11,  2.6193e-10,  1.3824e-10, -1.9791e-09,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  1.3642e-12,\n",
      "        -7.9694e-11,  4.0927e-12, -1.7462e-10,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  1.4552e-11,  5.6571e-10, -2.5830e-10,\n",
      "        -1.2860e-09,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -1.4552e-11,  6.6393e-11,  1.1278e-10, -6.5484e-11,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  6.5484e-11, -4.1109e-10,\n",
      "         9.6043e-10, -1.7462e-10, -9.0949e-13,  3.3879e-11, -2.4556e-11,\n",
      "         1.9099e-11, -9.0949e-12, -1.7167e-10,  8.5493e-11,  1.0914e-10,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00, -9.0949e-13, -2.2737e-13,\n",
      "        -1.8190e-12,  2.1828e-11,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -1.8190e-12,\n",
      "         7.2760e-12,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  2.5466e-11,  8.8858e-10, -8.0036e-11, -1.2406e-09,\n",
      "         2.1100e-10, -1.0841e-09, -1.0623e-09, -4.4674e-09,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  3.6380e-12,  0.0000e+00,  3.6380e-11,\n",
      "         2.1646e-10, -5.3114e-10,  1.0186e-10, -4.5475e-12,  1.3176e-10,\n",
      "         2.0918e-11,  9.4587e-11,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -1.3642e-12, -1.7963e-11,  8.1855e-12, -3.5016e-11,\n",
      "         1.0914e-11, -1.3370e-10, -6.9122e-11, -8.3855e-10,  0.0000e+00,\n",
      "        -3.7744e-11, -6.5484e-11, -3.6380e-12,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -2.1828e-11, -5.8208e-10,  1.3824e-10, -2.3283e-10,\n",
      "        -1.8190e-11,  6.2187e-11,  2.8649e-11, -3.2696e-10,  8.1855e-12,\n",
      "        -6.7757e-11,  5.2751e-11, -4.0109e-10,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00, -3.6380e-12,  4.5475e-12,  3.4561e-11,\n",
      "         1.5461e-10,  2.9104e-11,  2.7430e-09,  6.6939e-10, -3.4561e-09,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  9.0949e-13,\n",
      "        -2.2737e-13,  9.0949e-13,  0.0000e+00, -2.2737e-13,  0.0000e+00,\n",
      "        -4.5475e-13,  0.0000e+00,  2.2737e-12, -1.3551e-10,  2.2737e-12,\n",
      "        -1.2051e-10,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  1.0914e-11,\n",
      "        -2.8194e-11,  2.1828e-11,  1.2733e-10,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         1.4779e-12,  2.1828e-11, -1.2733e-11,  1.4552e-10,  2.9104e-11,\n",
      "         5.1296e-10, -1.3824e-10,  1.4334e-09,  5.8208e-11, -7.9126e-10,\n",
      "        -4.8021e-10,  1.7317e-09, -6.3665e-12,  5.5024e-11, -4.9113e-11,\n",
      "        -2.2919e-10, -7.2760e-11,  1.8863e-09, -5.2387e-10, -6.6866e-09,\n",
      "        -1.2733e-11, -3.6061e-10,  3.0923e-11,  2.2919e-10,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  1.8190e-11,  2.6284e-10, -7.2760e-12, -1.7280e-10,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  2.1828e-11,\n",
      "        -5.4479e-10, -3.6380e-12,  1.6080e-09, -1.0914e-11,  0.0000e+00,\n",
      "        -2.1828e-11, -1.4552e-11, -1.7462e-09,  3.2596e-09,  9.3132e-10,\n",
      "         0.0000e+00, -5.3551e-09,  9.3132e-10,  4.0745e-10,  0.0000e+00,\n",
      "         0.0000e+00, -1.9791e-09,  1.3970e-09,  0.0000e+00, -4.6566e-10,\n",
      "         0.0000e+00,  2.3283e-10, -7.1013e-09,  0.0000e+00, -5.0059e-09,\n",
      "         0.0000e+00, -9.3132e-10,  0.0000e+00, -5.8208e-09, -8.1491e-10,\n",
      "         0.0000e+00,  0.0000e+00, -4.8894e-09,  5.3551e-09,  0.0000e+00,\n",
      "         0.0000e+00, -5.8208e-11,  0.0000e+00,  0.0000e+00,  2.7940e-09,\n",
      "        -1.9791e-09,  0.0000e+00, -1.7462e-09,  0.0000e+00,  6.5193e-09,\n",
      "        -2.3283e-10, -1.2224e-09, -5.8208e-10,  1.7462e-10, -1.1642e-10,\n",
      "         0.0000e+00,  5.8208e-11,  0.0000e+00,  0.0000e+00,  2.3283e-09,\n",
      "         0.0000e+00,  2.3283e-10,  2.3283e-10,  0.0000e+00, -4.6566e-09,\n",
      "         4.1910e-09, -9.3132e-10,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  2.3283e-10,  5.8208e-11,  5.8208e-10,  3.4925e-09,\n",
      "         4.1910e-09,  9.3132e-10,  0.0000e+00,  2.3283e-10,  0.0000e+00,\n",
      "         2.9104e-10,  0.0000e+00, -6.9849e-10,  0.0000e+00,  1.8626e-09,\n",
      "        -1.1642e-10, -9.8953e-10,  0.0000e+00,  0.0000e+00,  1.1642e-10,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  2.3283e-10,\n",
      "         0.0000e+00,  1.1642e-10,  3.2596e-09,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  1.1642e-10,  4.1910e-09,  0.0000e+00,\n",
      "         0.0000e+00,  5.8208e-11, -1.9791e-09,  6.4028e-10,  0.0000e+00,\n",
      "         0.0000e+00, -2.3283e-10,  0.0000e+00,  3.2014e-10,  0.0000e+00,\n",
      "         1.7462e-10,  1.8626e-09,  0.0000e+00,  2.9104e-11,  0.0000e+00,\n",
      "         1.7462e-10,  0.0000e+00,  0.0000e+00, -4.6566e-10,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00, -5.8208e-11, -2.3283e-09,  4.5402e-09,\n",
      "        -7.5670e-10, -3.2596e-09,  1.3388e-09,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  1.3970e-09,  0.0000e+00, -1.3388e-09,  0.0000e+00,\n",
      "        -1.1991e-08, -4.6566e-10,  0.0000e+00, -7.2760e-12, -8.3819e-09,\n",
      "         1.8626e-09,  8.5493e-10,  1.8626e-09,  0.0000e+00, -1.1176e-08,\n",
      "        -1.1642e-09,  0.0000e+00,  1.6298e-09,  0.0000e+00, -2.3283e-09,\n",
      "         1.4435e-08,  0.0000e+00, -3.2596e-09,  0.0000e+00,  6.5193e-09,\n",
      "         0.0000e+00, -2.0489e-08, -1.3970e-09,  0.0000e+00,  0.0000e+00,\n",
      "        -2.7474e-08, -2.0489e-08,  0.0000e+00,  0.0000e+00, -4.6566e-10,\n",
      "         0.0000e+00,  0.0000e+00, -1.1642e-09,  2.7940e-09,  0.0000e+00,\n",
      "        -7.4506e-09,  0.0000e+00, -1.1176e-08,  2.0955e-08, -1.3970e-09,\n",
      "         8.2655e-09, -1.0710e-08,  1.8626e-09,  0.0000e+00,  1.3970e-09,\n",
      "         0.0000e+00,  0.0000e+00, -2.3283e-09,  0.0000e+00, -8.9640e-09,\n",
      "         4.6566e-10,  0.0000e+00,  4.2375e-08, -1.6298e-08,  2.5611e-09,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -1.5716e-09,\n",
      "         0.0000e+00, -3.7253e-09, -1.0245e-08, -1.6298e-08, -2.3647e-11,\n",
      "         0.0000e+00, -3.4925e-08,  0.0000e+00,  1.3970e-08,  0.0000e+00,\n",
      "        -2.7940e-09,  0.0000e+00,  9.3132e-10, -1.3970e-09,  6.9849e-10,\n",
      "         0.0000e+00,  0.0000e+00, -2.3283e-10,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00, -5.8208e-11,  0.0000e+00, -1.0710e-08,\n",
      "         4.6566e-10,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00, -4.7497e-08,  2.0955e-08,  0.0000e+00,  5.5879e-09,\n",
      "         1.1176e-08,  5.1223e-09,  0.0000e+00,  0.0000e+00, -8.3819e-09,\n",
      "        -1.8859e-08,  1.8626e-08,  0.0000e+00,  6.9849e-10,  2.3283e-09,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  3.8184e-08,  0.0000e+00,\n",
      "         0.0000e+00,  1.3970e-09,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "        -2.4447e-09,  4.6566e-09,  3.9581e-09,  9.3132e-09,  2.1420e-08,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  1.8626e-09,\n",
      "         0.0000e+00,  1.8161e-08,  6.9849e-10, -5.5879e-09, -2.3283e-10,\n",
      "         0.0000e+00,  0.0000e+00, -1.6298e-09,  9.3132e-10, -3.2414e-09,\n",
      "        -9.3132e-10,  0.0000e+00,  1.5018e-08,  0.0000e+00,  0.0000e+00,\n",
      "        -1.2806e-09,  0.0000e+00,  1.8626e-08, -1.5367e-08,  0.0000e+00,\n",
      "        -3.4925e-09,  0.0000e+00,  9.3132e-10,  0.0000e+00,  0.0000e+00,\n",
      "         9.7789e-09,  0.0000e+00,  0.0000e+00,  5.5879e-09, -2.2817e-08,\n",
      "         0.0000e+00,  0.0000e+00, -2.3283e-09,  0.0000e+00,  2.3283e-10,\n",
      "         1.8626e-09, -8.3819e-09,  0.0000e+00,  1.3970e-08,  0.0000e+00,\n",
      "         3.3528e-08, -2.4214e-08,  9.3132e-10, -5.3551e-09,  2.3283e-09,\n",
      "        -9.3132e-09, -6.9849e-10, -1.1642e-09,  0.0000e+00,  0.0000e+00,\n",
      "         2.5146e-08,  0.0000e+00, -9.5461e-09, -2.3283e-09,  0.0000e+00,\n",
      "        -4.6566e-08,  1.5832e-08, -2.0955e-09,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00, -4.0745e-10, -1.1642e-10,  9.3132e-09,\n",
      "         1.0710e-08, -1.3970e-09, -6.7848e-10,  0.0000e+00,  9.3132e-09,\n",
      "         0.0000e+00, -3.2596e-09,  0.0000e+00, -9.3132e-09,  0.0000e+00,\n",
      "         5.8208e-10, -4.6566e-10, -1.6298e-09,  0.0000e+00,  0.0000e+00,\n",
      "        -4.6566e-10,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         5.8208e-11,  0.0000e+00,  1.2107e-08, -1.7462e-09,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00, -2.6077e-08,\n",
      "        -8.8476e-09,  0.0000e+00,  3.7253e-09, -9.5461e-09, -1.6298e-09,\n",
      "         0.0000e+00,  0.0000e+00, -5.5879e-09,  3.4925e-09, -3.9116e-08,\n",
      "         0.0000e+00, -4.6566e-10, -4.1618e-09,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  7.9162e-09,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  0.0000e+00,  5.8208e-11, -3.7253e-09,\n",
      "        -1.3970e-09, -1.8626e-09, -1.3039e-08, -2.7940e-09,  0.0000e+00,\n",
      "         0.0000e+00,  0.0000e+00,  3.2596e-09,  0.0000e+00, -9.7789e-09,\n",
      "         4.6566e-10,  2.2352e-08, -3.7253e-09])\n"
     ]
    },
    {
     "ename": "ZeroDivisionError",
     "evalue": "division by zero",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mZeroDivisionError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-44-5b2919399dcc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     54\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-44-5b2919399dcc>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m()\u001b[0m\n\u001b[1;32m     47\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrads\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m         \u001b[0;36m1\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     50\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m100\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mZeroDivisionError\u001b[0m: division by zero"
     ]
    }
   ],
   "source": [
    "def train():\n",
    "    optimizer_td = torch.optim.Adam(model_td.parameters(), lr=1e-2)\n",
    "\n",
    "    #玩N局游戏,每局游戏训练一次\n",
    "    for epoch in range(1000):\n",
    "        #玩一局游戏,得到数据\n",
    "        states, rewards, actions, probs, next_states, overs = get_data()\n",
    "\n",
    "        #计算values和targets\n",
    "        values = [model_td(state) for state in states]\n",
    "        targets = [model_td(next_state) * 0.98 for next_state in next_states]\n",
    "        targets = [\n",
    "            target * (0 if over else 1)\n",
    "            for target, over in zip(targets, overs)\n",
    "        ]\n",
    "        targets = [target + reward for target, reward in zip(targets, rewards)]\n",
    "\n",
    "        #第一部分的loss是针对value的,越接近target就越好\n",
    "        loss_td = [\n",
    "            torch.nn.functional.mse_loss(value, target.detach())\n",
    "            for value, target in zip(values, targets)\n",
    "        ]\n",
    "        loss_td = sum(loss_td) / len(loss_td)\n",
    "\n",
    "        #更新参数\n",
    "        optimizer_td.zero_grad()\n",
    "        loss_td.backward()\n",
    "        optimizer_td.step()\n",
    "\n",
    "        advantages = get_advantages([\n",
    "            (value - target).item() for value, target in zip(values, targets)\n",
    "        ])\n",
    "\n",
    "        #取出每一步的动作概率,取对数后作为loss\n",
    "        #因为概率一定是0-1之间的小数,所以取对数之后一定是负数\n",
    "        #符号取反,一定是正数,最终导致的结果就是,概率越小,loss越大,反之亦然\n",
    "        #这样会迫使prob的取值越来越大\n",
    "        old_action_prob_logs = [\n",
    "            prob[0, action].log() for prob, action in zip(probs, actions)\n",
    "        ]\n",
    "\n",
    "        surrogate_obj = compute_surrogate_obj(states, actions, advantages,\n",
    "                                              old_action_prob_logs, model)\n",
    "        \n",
    "        grads = torch.autograd.grad(surrogate_obj, model.parameters())\n",
    "        grads = torch.cat([grad.view(-1) for grad in grads]).detach()\n",
    "        print(grads)\n",
    "        \n",
    "        1 / 0\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            test_result = sum([test(play=False) for _ in range(10)]) / 10\n",
    "            print(epoch, test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAATVElEQVR4nO3dbYyd5Z3f8e8PYyCCVEA9EK8fajfrrQrRrtmM3DRUFU2yi8uu6qAqlSMt8gskpxJRE3WVFnarbvLC0na1eXjTRCINXSebjeMqUCyUtsvSRCjKFseEh2DAYIIbJnawccrykMb44d8Xc1uc2mP7eM7cM75mvh/p6Nznf9/3Of8Le36+uM595qSqkCS146K5bkCSdH4MbklqjMEtSY0xuCWpMQa3JDXG4JakxvQW3EnWJ9mTZG+SO/t6HUlaaNLHddxJFgHPAb8FTAA/AD5aVU/P+ItJ0gLT14x7HbC3qn5cVW8B24ANPb2WJC0oF/f0vMuAlwYeTwD/4EwHL1mypFatWtVTK5LUnn379vHKK69kqn19BfdUL/b/rckk2QxsBli5ciW7du3qqRVJas/4+PgZ9/W1VDIBrBh4vBzYP3hAVd1dVeNVNT42NtZTG5I0//QV3D8A1iRZneQSYCOwo6fXkqQFpZelkqo6luTjwP8AFgH3VNXuPl5Lkhaavta4qapvA9/u6/klaaHyk5OS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhoz0leXJdkHvA4cB45V1XiSq4FvAquAfcC/qKr/M1qbkqSTZmLG/U+qam1VjXeP7wQeqqo1wEPdY0nSDOljqWQDsLXb3gp8uIfXkKQFa9TgLuAvkzyaZHNXu7aqDgB099eM+BqSpAEjrXEDN1bV/iTXAA8meXbYE7ug3wywcuXKEduQpIVjpBl3Ve3v7g8C9wHrgJeTLAXo7g+e4dy7q2q8qsbHxsZGaUOSFpRpB3eSy5O88+Q28NvAU8AOYFN32Cbg/lGblCS9bZSlkmuB+5KcfJ6/qKr/nuQHwPYktwM/AT4yepuSpJOmHdxV9WPgN6aoHwY+OEpTkqQz85OTktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmPOGdxJ7klyMMlTA7WrkzyY5Pnu/qqBfXcl2ZtkT5Kb+2pckhaqYWbcfwasP6V2J/BQVa0BHuoek+Q6YCNwfXfOF5MsmrFuJUnnDu6qehj4+SnlDcDWbnsr8OGB+raqOlJVLwJ7gXUz06okCaa/xn1tVR0A6O6v6erLgJcGjpvoaqdJsjnJriS7Dh06NM02JGnhmek3JzNFraY6sKrurqrxqhofGxub4TYkaf6abnC/nGQpQHd/sKtPACsGjlsO7J9+e5KkU003uHcAm7rtTcD9A/WNSS5NshpYA+wcrUVJ0qCLz3VAkm8ANwFLkkwAfwT8MbA9ye3AT4CPAFTV7iTbgaeBY8AdVXW8p94laUE6Z3BX1UfPsOuDZzh+C7BllKYkSWfmJyclqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY0xuCWpMQa3JDXmnMGd5J4kB5M8NVD7dJKfJnm8u90ysO+uJHuT7Elyc1+NS9JCNcyM+8+A9VPUP19Va7vbtwGSXAdsBK7vzvlikkUz1awkaYjgrqqHgZ8P+XwbgG1VdaSqXgT2AutG6E+SdIpR1rg/nuTJbinlqq62DHhp4JiJrnaaJJuT7Eqy69ChQyO0IUkLy3SD+0vAu4G1wAHgs109UxxbUz1BVd1dVeNVNT42NjbNNiRp4ZlWcFfVy1V1vKpOAF/m7eWQCWDFwKHLgf2jtShJGjSt4E6ydODhrcDJK052ABuTXJpkNbAG2Dlai5KkQRef64Ak3wBuApYkmQD+CLgpyVoml0H2AR8DqKrdSbYDTwPHgDuq6ngvnUvSAnXO4K6qj05R/spZjt8CbBmlKUnSmfnJSUlqjMEtSY0xuCWpMQa3JDXG4JakxpzzqhLpTKqKNw/t48TRI6ftu/yaVSxafNkcdCXNfwa3RvK/H/5zfnH4pdPq1/3zf8flS1bOQUfS/OdSifpRU/6KGkkzwOBWPwxuqTcGt3pRBrfUG4NbPTkx1w1I85bBrV4445b6Y3CrHwa31BuDW71wxi31x+BWTwxuqS8Gt/rhjFvqjcGtXkx+HamkPhjc6oczbqk35wzuJCuSfCfJM0l2J/lEV786yYNJnu/urxo4564ke5PsSXJznwPQhck3J6X+DDPjPgb8flX9feB9wB1JrgPuBB6qqjXAQ91jun0bgeuB9cAXkyzqo3ldwAxuqTfnDO6qOlBVP+y2XweeAZYBG4Ct3WFbgQ932xuAbVV1pKpeBPYC62a4b13gyk9OSr05rzXuJKuAG4BHgGur6gBMhjtwTXfYMmDw93xOdLVTn2tzkl1Jdh06dGgareuC5oxb6s3QwZ3kCuBbwCer6rWzHTpF7bSf4qq6u6rGq2p8bGxs2DbUCoNb6s1QwZ1kMZOh/fWqurcrv5xkabd/KXCwq08AKwZOXw7sn5l21QrfnJT6M8xVJQG+AjxTVZ8b2LUD2NRtbwLuH6hvTHJpktXAGmDnzLWsJhjcUm+G+eqyG4HbgB8lebyr/QHwx8D2JLcDPwE+AlBVu5NsB55m8oqUO6rq+Ew3rgudb05KfTlncFfV95h63Rrgg2c4ZwuwZYS+1DiXSqT++MlJ9cPglnpjcKsXzril/hjc6oe/ZErqjcGtXpS/j1vqjcGtfrhUIvXG4FYvXOOW+mNwqx8Gt9Qbg1v98M1JqTcGt3rhm5NSfwxu9cOlEqk3Brd64ZuTUn8MbvXD4JZ6Y3BrNJn694/VCX8hpNQXg1sjueJdvzpl/Y2f7Z3lTqSFw+DWSC66+JIp6yeOH53lTqSFw+DWSHKGpRJJ/TG4NSL/CkmzzZ86jcQZtzT7hvmy4BVJvpPkmSS7k3yiq386yU+TPN7dbhk4564ke5PsSXJznwPQHDO4pVk3zJcFHwN+v6p+mOSdwKNJHuz2fb6q/nTw4CTXARuB64FfAf4qya/5hcHzlMEtzbpzzrir6kBV/bDbfh14Blh2llM2ANuq6khVvQjsBdbNRLO68LhUIs2+81rjTrIKuAF4pCt9PMmTSe5JclVXWwa8NHDaBGcPejXN4JZm29DBneQK4FvAJ6vqNeBLwLuBtcAB4LMnD53i9NM+/5xkc5JdSXYdOnTofPvWBSLx/W1ptg31U5dkMZOh/fWquhegql6uquNVdQL4Mm8vh0wAKwZOXw7sP/U5q+ruqhqvqvGxsbFRxqC55FKJNOuGuaokwFeAZ6rqcwP1pQOH3Qo81W3vADYmuTTJamANsHPmWtaFxDVuafYNc1XJjcBtwI+SPN7V/gD4aJK1TC6D7AM+BlBVu5NsB55m8oqUO7yiZB4zuKVZd87grqrvMfW69bfPcs4WYMsIfakVrnFLs86fOo3EpRJp9hncGpHBLc02g1sjccYtzT6DW6MxuKVZZ3BrNL45Kc06f+o0EpdKpNlncGtEBrc02wxujcQZtzT7DG6NxjVuadb5U6fROOOWZp3BrZHENW5p1hncGo0zbmnWDfPbAbXAvPnmmzz55JNUnfb9F6fJ3+yb8i/Rq6++yve///2hXm9sbIw1a9acZ5fSwmVw6zTPPfcc73//+4c69qa1q/iTf/lbp9V3PrKTf/V7nxnqOW677Ta++tWvnleP0kJmcGskVcWJCofeWsHho8u49KJfsPyy5+a6LWleM7g1kuMF+/7vr/PcL8YpLgKKl99azZETp31bnaQZ4puTGsnP33pXF9qLmPwU5UX8zbExnn3zH851a9K8ZXBrJMdOXNSF9qBwrBbPST/SQjDMlwVflmRnkieS7E7yma5+dZIHkzzf3V81cM5dSfYm2ZPk5j4HoLm1iF9ycY6cUi3ecdEbc9KPtBAMM+M+Anygqn4DWAusT/I+4E7goapaAzzUPSbJdcBG4HpgPfDFJKdOyTRP/K1FB3nPFQ9zSX4BFBdxlHdd8mP+3uX/a65bk+atYb4suICT06fF3a2ADcBNXX0r8F3g33b1bVV1BHgxyV5gHfDXZ3qNo0eP8rOf/Wx6I9CMO3z48NDHvnjgVf7zf/kqrx9/gNeOLWFxjrDkkgleefW1oZ/jl7/8pX/+0imOHj16xn1DXVXSzZgfBX4V+I9V9UiSa6vqAEBVHUhyTXf4MmBwujXR1c7o8OHDfO1rXxumFc2C/fuHvyLk4Ktv8l+/9+xIr/fCCy/45y+d4mwTqKGCu6qOA2uTXAncl+Q9Zzl8qs9An/YRvCSbgc0AK1eu5FOf+tQwrWgWPPbYY3zhC1+Ytde7/vrr/fOXTvHNb37zjPvO66qSqnqVySWR9cDLSZYCdPcHu8MmgBUDpy0HTpvCVdXdVTVeVeNjY2Pn04YkLWjDXFUy1s20SfIO4EPAs8AOYFN32Cbg/m57B7AxyaVJVgNrgJ0z3LckLVjDLJUsBbZ269wXAdur6oEkfw1sT3I78BPgIwBVtTvJduBp4BhwR7fUIkmaAcNcVfIkcMMU9cPAB89wzhZgy8jdSZJO4ycnJakxBrckNcbfDqjTXHnlldx6661DfZHCTHjve987K68jzRcGt06zevVq7r333rluQ9IZuFQiSY0xuCWpMQa3JDXG4JakxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhozzJcFX5ZkZ5InkuxO8pmu/ukkP03yeHe7ZeCcu5LsTbInyc19DkCSFpphfh/3EeADVfVGksXA95L8t27f56vqTwcPTnIdsBG4HvgV4K+S/JpfGCxJM+OcM+6a9Eb3cHF3O9tXo2wAtlXVkap6EdgLrBu5U0kSMOQad5JFSR4HDgIPVtUj3a6PJ3kyyT1Jrupqy4CXBk6f6GqSpBkwVHBX1fGqWgssB9YleQ/wJeDdwFrgAPDZ7vBM9RSnFpJsTrIrya5Dhw5No3VJWpjO66qSqnoV+C6wvqpe7gL9BPBl3l4OmQBWDJy2HNg/xXPdXVXjVTU+NjY2nd4laUEa5qqSsSRXdtvvAD4EPJtk6cBhtwJPdds7gI1JLk2yGlgD7JzRriVpARvmqpKlwNYki5gM+u1V9UCSryVZy+QyyD7gYwBVtTvJduBp4Bhwh1eUSNLMOWdwV9WTwA1T1G87yzlbgC2jtSZJmoqfnJSkxhjcktQYg1uSGmNwS1JjDG5JaozBLUmNMbglqTEGtyQ1xuCWpMYY3JLUGINbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTGGNyS1BiDW5IaY3BLUmMMbklqjMEtSY1JVc11DyQ5BLwJvDLXvfRgCY6rNfN1bI6rLX+nqsam2nFBBDdAkl1VNT7Xfcw0x9We+To2xzV/uFQiSY0xuCWpMRdScN891w30xHG1Z76OzXHNExfMGrckaTgX0oxbkjSEOQ/uJOuT7EmyN8mdc93P+UpyT5KDSZ4aqF2d5MEkz3f3Vw3su6sb654kN89N1+eWZEWS7yR5JsnuJJ/o6k2PLcllSXYmeaIb12e6etPjOinJoiSPJXmgezxfxrUvyY+SPJ5kV1ebF2OblqqasxuwCHgB+LvAJcATwHVz2dM0xvCPgd8Enhqo/QlwZ7d9J/Afuu3rujFeCqzuxr5orsdwhnEtBX6z234n8FzXf9NjAwJc0W0vBh4B3tf6uAbG96+BvwAemC9/F7t+9wFLTqnNi7FN5zbXM+51wN6q+nFVvQVsAzbMcU/npaoeBn5+SnkDsLXb3gp8eKC+raqOVNWLwF4m/xtccKrqQFX9sNt+HXgGWEbjY6tJb3QPF3e3ovFxASRZDvwO8J8Gys2P6yzm89jOaq6Dexnw0sDjia7Wumur6gBMBiBwTVdvcrxJVgE3MDk7bX5s3XLC48BB4MGqmhfjAr4A/BvgxEBtPowLJv9x/cskjybZ3NXmy9jO28Vz/PqZojafL3NpbrxJrgC+BXyyql5LphrC5KFT1C7IsVXVcWBtkiuB+5K85yyHNzGuJL8LHKyqR5PcNMwpU9QuuHENuLGq9ie5BngwybNnOba1sZ23uZ5xTwArBh4vB/bPUS8z6eUkSwG6+4NdvanxJlnMZGh/varu7crzYmwAVfUq8F1gPe2P60bgnyXZx+SS4weS/DntjwuAqtrf3R8E7mNy6WNejG065jq4fwCsSbI6ySXARmDHHPc0E3YAm7rtTcD9A/WNSS5NshpYA+ycg/7OKZNT668Az1TV5wZ2NT22JGPdTJsk7wA+BDxL4+OqqruqanlVrWLy5+h/VtXv0fi4AJJcnuSdJ7eB3waeYh6Mbdrm+t1R4BYmr1h4AfjDue5nGv1/AzgAHGXyX/rbgb8NPAQ8391fPXD8H3Zj3QP807nu/yzj+kdM/u/lk8Dj3e2W1scG/DrwWDeup4B/39WbHtcpY7yJt68qaX5cTF519kR3230yJ+bD2KZ785OTktSYuV4qkSSdJ4NbkhpjcEtSYwxuSWqMwS1JjTG4JakxBrckNcbglqTG/D83EMnFN8dFZAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test(play=True)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "第9章-策略梯度算法.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
